Reproducible Documentation of Analysis Study 1

Import of the Data

The data was assessed with the formr survey framework (Arslan, Walther, and Tata 2020). The raw data was imported via the following code.

library(tidyverse)

── Attaching packages ─────────────────────────────────────── tidyverse 1.3.2 ──
✔ ggplot2 3.4.0      ✔ purrr   0.3.5 
✔ tibble  3.1.8      ✔ dplyr   1.0.10
✔ tidyr   1.2.1      ✔ stringr 1.4.1 
✔ readr   2.1.3      ✔ forcats 0.5.2 
── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag()    masks stats::lag()

library(ggforce)
library(lme4)

Lade nötiges Paket: Matrix

Attache Paket: 'Matrix'

Die folgenden Objekte sind maskiert von 'package:tidyr':

    expand, pack, unpack

library(BFpack)

Lade nötiges Paket: bain
Registered S3 methods overwritten by 'BFpack':
  method               from
  get_estimates.lm     bain
  get_estimates.t_test bain

library(hrbrthemes)
library(patchwork)
library(brms)

Lade nötiges Paket: Rcpp
Loading 'brms' package (version 2.18.0). Useful instructions
can be found by typing help('brms'). A more detailed introduction
to the package is available through vignette('brms_overview').

Attache Paket: 'brms'

Das folgende Objekt ist maskiert 'package:lme4':

    ngrps

Das folgende Objekt ist maskiert 'package:stats':

    ar

library(viridis)

Lade nötiges Paket: viridisLite

load("../data/teachers_study1_N40.RData")

set.seed(25051982)

Data Wrangling

# wrangle information on the plot type, ES, ...
plot_info <- study1 %>%
    pivot_longer(2:195, names_to = "variables", values_to = "values", 
                 values_transform = as.character) %>%
    dplyr::filter(str_detect(variables, "plot")) %>% 
    # we only need the rows with info on plots
    tidyr::separate(col = values, into = c("type", "axis", "effsize"), 
                    # separate the info into three columns
                    sep = "_", remove = F) %>%
    dplyr::mutate(plot = variables,       # rename variables for later join
                  type = paste(type, axis, sep = "_")) %>%
    dplyr::select(-variables, -axis)

# wrangle answers to items on each page
item_values <- study1 %>%
    dplyr::select(-c(topic:itemo)) %>%
    pivot_longer(2:169, names_to = "variables", values_to = "values", 
                 values_transform = as.character) %>%
    dplyr::mutate(variables = case_when(      # recode variable names that have
        variables == "sensi_6" ~ "sensi_06",  # accidentally been labeled
        variables == "acccl_6" ~ "acccl_06",  # without zero
        variables == "accu3_6" ~ "accu3_06",
        variables == "accov_6" ~ "accov_06",
        variables == "diffi_6" ~ "diffi_06",
        variables == "infor_6" ~ "infor_06",
        variables == "value_6" ~ "value_06",
        TRUE ~ variables 
    )) %>%
    dplyr::mutate(plot = paste0("plotx_", str_sub(variables, -2, -1)), 
                  # create variable for later join
                  variables = str_sub(variables, 1, -4)) %>%    
    # rename variable names to get a data set 
    # with one line per participant per page
    pivot_wider(id_cols = c(session, plot), names_from = "variables", 
                values_from = "values")


# join the two data sets
study1_w <- full_join(plot_info, item_values, 
                               by = c("session", "plot")) %>% 
    # by participant and page (plot)
    dplyr::select(-values) %>%
    dplyr::mutate(rating_cl = as.numeric(acccl), # some var need to be defined as
                  rating_u3 = as.numeric(accu3), # numeric again
                  rating_ov = as.numeric(accov),
                  diffi = as.numeric(diffi),
                  infor = as.numeric(infor),
                  value = as.numeric(value),
                  effsize = as.numeric(effsize),
                  effsize_cl = case_when( 
                  # there is no negative Cliff's Delta, so we have to compute 
                  # two transformations
                      effsize > 0 ~   (((2*pnorm(effsize/2))-1)/pnorm(effsize/2)),
                  # transform the actual effect size Cohen's d to Cliff's Delta
                      effsize < 0 ~ (- (((2*pnorm(abs(effsize)/2))-1)/pnorm(abs(effsize)/2))) 
                  # transform the actual effect size Cohen's d to Cliff's Delta 
                  # and make it negative as in the item
                  ),
                  effsize_u3 = pnorm(effsize), 
                  # transform the actual effect size Cohen's d to Cohen's U3
                  effsize_ov = 2 * pnorm(-abs(effsize) / 2), 
                  # transform the actual effect size Cohen's d to overlap
                  # actual difference of rating relative to depicted effectsize 
                  diff_cl = (rating_cl - effsize_cl)/2,
                  # actual difference of rating relative to depicted effectsize
                  diff_u3 = (rating_u3/100) - effsize_u3,
                  # actual difference of rating relative to depicted effectsize 
                  diff_ov = (rating_ov/100) - effsize_ov,
                  diffi_normed = ((diffi - 1)  / 3) - 1, # transform item to -1 to 1
                  infor_normed = ((infor - 1)  / 3) - 1, # transform item to -1 to 1
                  value_normed = ((value - 1)  / 3) - 1) %>%  # transform item to -1 to 1
    group_by(session) %>% 
    mutate(rating_ov_missconcept = median(rating_ov, na.rm = T) < 68,
           rating_u3_missconcept = median(rating_u3, na.rm = T) < 21.2) %>% 
    ungroup() %>% 
    mutate(rating_u3_filtered = ifelse(rating_u3_missconcept == T, NA, rating_u3),
           rating_ov_filtered = ifelse(rating_ov_missconcept == T, NA, rating_ov),
           diff_u3_filtered = (rating_u3_filtered/100) - effsize_u3,
           diff_ov_filtered = (rating_ov_filtered/100) - effsize_ov,
           sensi_binary = ifelse(is.na(sensi), # 1 if NOT "equal"
                                        NA,
                                        as.numeric(!grepl("equal", sensi))),
           sensi_ordinal = ordered(factor(substr(sensi, 55, 100)),
                                   levels = c("inferior",
                                              "equal",
                                              "superior")),
           sensi_binary_filtered = case_when(sensi_ordinal == "equal" ~ 0, 
                                              (sensi_ordinal == "inferior" & 
                                                  effsize < 0) | 
                                                  (sensi_ordinal == "superior" & 
                                                  effsize > 0) ~ as.numeric(NA),
                                              TRUE ~ 1),
           effsize_abs = abs(effsize))

# create a list of u3_misconceptualizers
u3_misconceptualizers <-
    study1_w %>% 
    filter(rating_u3_missconcept == T) %>% 
    pull(session) %>% 
    unique()

# create a list of ov_misconceptualizers
ov_misconceptualizers <-
    study1_w %>% 
    filter(rating_ov_missconcept == T) %>% 
    pull(session) %>% 
    unique()    

### wrangle time stamp data ####################################################
study1_w_timestamp <- 
    read_csv("../data/teachers_study1_N40_detailed.csv") %>% 
    # filter participants from study1_w only
    filter(session %in% study1_w$session) %>% 
    # we only need vars sensitivity or accuracy
    dplyr::filter(str_detect(item_name, "sensi|acccl|accu3|accov")) %>%  
    # create var with plot number
    mutate(plot = paste0("plotx_", str_sub(item_name, -2, -1)),
           # recode wrong item labelling
           plot = ifelse(plot == "plotx__6", "plotx_06", plot)) %>% 
    relocate(session, plot) %>% 
    # delete the page number in item name
    mutate(item_name = str_sub(item_name, 1, 5)) %>%  
    pivot_wider(id_cols = c(session, plot), names_from = item_name, 
                values_from = answered_relative) %>% 
    rowwise() %>%
    # what was the time of the first item to be clicked?
    mutate(effic = min(sensi, acccl, accu3, accov, na.rm=T)) %>%
    ungroup() %>% 
    dplyr::select(session, plot, effic, sensi, acccl, accu3, accov) %>% 
    left_join(., study1_w %>% 
                  select(session, plot, type), by=c("session", "plot")) %>% 
    # generate data set so that the six plots from the same type are ordered
    # one after the other (and not 1-24)
    group_by(session, type) %>% 
    arrange(plot) %>% 
    mutate(plotNrWithin = 1:n()) %>%
    ungroup() %>% 
    group_by(plotNrWithin, type) %>% 
    mutate(effic_10righttrunc = ifelse(effic > quantile(effic, .9), NA, effic),
           effic_05righttrunc = ifelse(effic > quantile(effic, .95), NA, effic),
           log_effic_05righttrunc =log(effic_05righttrunc),
           log_effic_10righttrunc = log(effic_10righttrunc),
           plotNrWithin0 = plotNrWithin -1,
           plotNrWithin_factor = as.factor(plotNrWithin)) %>% 
    ungroup()

Rows: 27336 Columns: 14
── Column specification ────────────────────────────────────────────────────────
Delimiter: ","
chr  (3): session, item_name, answer
dbl  (7): unit_session_id, item_id, shown_relative, answered_relative, displ...
dttm (4): created, saved, shown, answered

ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.

Accuracy

Distribution of the Dependent Variables

study1_w %>% 
    select(rating_cl, rating_u3, rating_ov) %>% 
    pivot_longer(
        c(rating_cl, rating_u3, rating_ov),
        names_to = "dependent_variable", 
        values_to = "rated_effectsize"
        ) %>% 
    ggplot(., aes(rated_effectsize)) +
    geom_histogram() +
    facet_wrap(~dependent_variable, scales = "free_x") +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white"))

`stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

Warning: Removed 1440 rows containing non-finite values (`stat_bin()`).

Somewhat disturbing is the first mode in accov. Maybe some users confused overlap and non-overlap? Another artefact seems to be the first mode in accu3.

Are there Constant Misconceptions per Persons?

ggplot(study1_w %>% 
    select(rating_cl, rating_u3, rating_ov, effsize, effsize_cl, session) %>% 
    pivot_longer(
        c(rating_cl, rating_u3, rating_ov),
        names_to = "operationalization", 
        values_to = "judged_effectsize"
        ),
    aes(judged_effectsize, as.numeric(as.factor(session)),
        color = session)
        ) +
    geom_jitter(height = 0) +
    facet_wrap(~ operationalization, scales = "free_x") +
    theme_modern_rc() +
    theme(legend.position = "none",
          strip.text = element_text(color = "white"))

Warning: Removed 1440 rows containing missing values (`geom_point()`).

It seems to be so, as ….

Association of Ratings and Actual Effect Size

study1_w %>% 
    ggplot(., aes(effsize, rating_cl)) +
    geom_jitter() +
    stat_smooth() +
    theme_modern_rc()

`geom_smooth()` using method = 'loess' and formula = 'y ~ x'

Warning: Removed 480 rows containing non-finite values (`stat_smooth()`).

Warning: Removed 480 rows containing missing values (`geom_point()`).

study1_w %>% 
    ggplot(., aes(effsize, rating_u3, color = rating_u3_missconcept)) +
    geom_jitter() +
    stat_smooth() +
    theme_modern_rc()

`geom_smooth()` using method = 'loess' and formula = 'y ~ x'

Warning: Removed 480 rows containing non-finite values (`stat_smooth()`).
Removed 480 rows containing missing values (`geom_point()`).

study1_w %>% 
    ggplot(., aes(abs(effsize), rating_ov, color = rating_ov_missconcept)) +
    geom_jitter() +
    stat_smooth(method = "lm") +
    theme_modern_rc()

`geom_smooth()` using formula = 'y ~ x'

Warning: Removed 480 rows containing non-finite values (`stat_smooth()`).
Removed 480 rows containing missing values (`geom_point()`).

The correlations undermine the interpretation of the misconceptions Therefor we will look at the intercorrelations of the dependent variables.

Compute Kendall’s \(\tau\) for accl, accu3 and accov

study1_w %>% 
    group_by(type) %>% 
    do(tau_cl = unlist(cor(.$effsize, .$rating_cl, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_cl)

# A tibble: 4 × 2
  type                tau_cl
  <chr>                <dbl>
1 gardneraltman_xaxis  0.373
2 halfeye_xaxis        0.548
3 halfeye_yaxis        0.484
4 raincloud_yaxis      0.415

study1_w %>% 
    group_by(rating_ov_missconcept, type) %>% 
    do(tau_ov = unlist(cor(abs(.$effsize), .$rating_ov, method = "kendall", 
                           use = "pairwise.complete"))) %>% 
    unnest(tau_ov)

# A tibble: 8 × 3
  rating_ov_missconcept type                 tau_ov
  <lgl>                 <chr>                 <dbl>
1 FALSE                 gardneraltman_xaxis -0.355 
2 FALSE                 halfeye_xaxis       -0.384 
3 FALSE                 halfeye_yaxis       -0.478 
4 FALSE                 raincloud_yaxis     -0.523 
5 TRUE                  gardneraltman_xaxis  0.0236
6 TRUE                  halfeye_xaxis        0.119 
7 TRUE                  halfeye_yaxis        0.0205
8 TRUE                  raincloud_yaxis     -0.220 

study1_w %>% 
    group_by(rating_u3_missconcept, type) %>% 
    do(tau_u3 = cor(.$effsize, .$rating_u3, method = "kendall", 
                           use = "pairwise.complete")) %>% 
    unnest(tau_u3)

# A tibble: 8 × 3
  rating_u3_missconcept type                 tau_u3
  <lgl>                 <chr>                 <dbl>
1 FALSE                 gardneraltman_xaxis -0.298 
2 FALSE                 halfeye_xaxis       -0.471 
3 FALSE                 halfeye_yaxis       -0.263 
4 FALSE                 raincloud_yaxis     -0.321 
5 TRUE                  gardneraltman_xaxis  0.119 
6 TRUE                  halfeye_xaxis       -0.153 
7 TRUE                  halfeye_yaxis       -0.108 
8 TRUE                  raincloud_yaxis     -0.0971

Associations of the Dependent Variables

ggplot(study1_w, 
       aes(rating_cl, rating_u3)) +
    geom_jitter() +
    theme_modern_rc() +
ggplot(study1_w, 
       aes(rating_cl, rating_ov)) +
    geom_jitter() +
    theme_modern_rc() +
ggplot(study1_w, 
       aes(rating_ov, rating_u3)) +
    geom_jitter() +
    theme_modern_rc()

Warning: Removed 480 rows containing missing values (`geom_point()`).
Removed 480 rows containing missing values (`geom_point()`).
Removed 480 rows containing missing values (`geom_point()`).

Global Under- or Overestimation

study1_w %>% 
    select(diff_cl, diff_u3_filtered, diff_ov_filtered, session, type) %>% 
    gather(dependent_variable, difference_to_true_effsize, 
           diff_cl, diff_u3_filtered, diff_ov_filtered) %>% 
    ggplot(., aes(type, difference_to_true_effsize)) +
    geom_boxplot(alpha = .3) +
    facet_wrap(~dependent_variable, scales = "free_y") +
    geom_jitter(aes(color = type)) +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white"),
          axis.text.x = element_blank())

Warning: Removed 1812 rows containing non-finite values (`stat_boxplot()`).

Warning: Removed 1812 rows containing missing values (`geom_point()`).

Intraclasscorrleations

icc_mod_rating_cl <- lmer(rating_cl ~  1 + (1|type), 
                          data = study1_w )

boundary (singular) fit: see help('isSingular')

icc_mod_rating_u3 <- lmer(rating_u3 ~  1 + (1|type), 
                          data = study1_w %>% 
                              filter(rating_u3_missconcept == F))
icc_mod_rating_ov <- lmer(rating_ov ~  1 + (1|type), 
                          data = study1_w %>% 
                              filter(rating_ov_missconcept == F))
icc_mod_diffi <- lmer(diffi ~  1 + (1|type), 
                      data = study1_w)
icc_mod_infor <- lmer(infor ~  1 + (1|type), 
                      data = study1_w )
icc_mod_value <- lmer(value ~  1 + (1|type), 
                      data = study1_w )

sjPlot::tab_model(
    icc_mod_rating_cl,
    icc_mod_rating_u3,
    icc_mod_rating_ov,
    icc_mod_diffi,
    icc_mod_infor,
    icc_mod_value
)

	rating_cl			rating_u3			rating_ov			diffi			infor			value
Predictors	Estimates	CI	p	Estimates	CI	p	Estimates	CI	p	Estimates	CI	p	Estimates	CI	p	Estimates	CI	p
(Intercept)	0.01	-0.02 – 0.05	0.391	43.08	39.89 – 46.27	<0.001	82.60	80.85 – 84.34	<0.001	4.05	3.63 – 4.47	<0.001	4.11	3.86 – 4.36	<0.001	4.13	3.87 – 4.39	<0.001
Random Effects
σ2	0.15			317.10			282.66			2.79			2.07			2.17
τ00	0.00 type			4.61 type			0.10 type			0.17 type			0.06 type			0.06 type
ICC				0.01			0.00			0.06			0.03			0.03
N	4 type			4 type			4 type			4 type			4 type			4 type
Observations	480			216			372			960			960			960
Marginal R2 / Conditional R2	0.000 / NA			0.000 / 0.014			0.000 / 0.000			0.000 / 0.058			0.000 / 0.026			0.000 / 0.028

BF(icc_mod_rating_cl, hypothesis = "icc = 0; icc > 0")

First, unconstrained icc analysis...

Second, exploratory testing of icc's...
icc; 

Third, confirmatory testing of icc's...
icc=0; 
icc>0; 

Call:
BF.lmerMod(x = icc_mod_rating_cl, hypothesis = "icc = 0; icc > 0")

Bayesian hypothesis test
Type: confirmatory
Object: lmerMod
Parameter: intraclass correlations
Method: Bayes factors based on uniform priors

Posterior probabilities:
   Pr(hypothesis|data)
H1               0.375
H2               0.006
H3               0.619

Evidence matrix (Bayes factors):
      H1     H2    H3
H1 1.000 59.126 0.605
H2 0.017  1.000 0.010
H3 1.652 97.673 1.000

Hypotheses:
H1: icc=0
H2: icc>0
H3: complement

BF(icc_mod_rating_u3, hypothesis = "icc = 0; icc > 0")

First, unconstrained icc analysis...

Second, exploratory testing of icc's...
icc; 

Third, confirmatory testing of icc's...
icc=0; 
icc>0; 

Call:
BF.lmerMod(x = icc_mod_rating_u3, hypothesis = "icc = 0; icc > 0")

Bayesian hypothesis test
Type: confirmatory
Object: lmerMod
Parameter: intraclass correlations
Method: Bayes factors based on uniform priors

Posterior probabilities:
   Pr(hypothesis|data)
H1               0.657
H2               0.108
H3               0.235

Evidence matrix (Bayes factors):
      H1    H2    H3
H1 1.000 6.075 2.793
H2 0.165 1.000 0.460
H3 0.358 2.175 1.000

Hypotheses:
H1: icc=0
H2: icc>0
H3: complement

BF(icc_mod_rating_ov, hypothesis = "icc = 0; icc > 0")

First, unconstrained icc analysis...

Second, exploratory testing of icc's...
icc; 

Third, confirmatory testing of icc's...
icc=0; 
icc>0; 

Call:
BF.lmerMod(x = icc_mod_rating_ov, hypothesis = "icc = 0; icc > 0")

Bayesian hypothesis test
Type: confirmatory
Object: lmerMod
Parameter: intraclass correlations
Method: Bayes factors based on uniform priors

Posterior probabilities:
   Pr(hypothesis|data)
H1               0.605
H2               0.027
H3               0.368

Evidence matrix (Bayes factors):
      H1     H2    H3
H1 1.000 22.432 1.648
H2 0.045  1.000 0.073
H3 0.607 13.615 1.000

Hypotheses:
H1: icc=0
H2: icc>0
H3: complement

BF(icc_mod_diffi, hypothesis = "icc = 0; icc > 0")

First, unconstrained icc analysis...

Second, exploratory testing of icc's...
icc; 

Third, confirmatory testing of icc's...
icc=0; 
icc>0; 

Call:
BF.lmerMod(x = icc_mod_diffi, hypothesis = "icc = 0; icc > 0")

Bayesian hypothesis test
Type: confirmatory
Object: lmerMod
Parameter: intraclass correlations
Method: Bayes factors based on uniform priors

Posterior probabilities:
   Pr(hypothesis|data)
H1                   0
H2                   1
H3                   0

Evidence matrix (Bayes factors):
        H1 H2  H3
H1       1  0 Inf
H2 7895605  1 Inf
H3       0  0   1

Hypotheses:
H1: icc=0
H2: icc>0
H3: complement

BF(icc_mod_infor, hypothesis = "icc = 0; icc > 0")

First, unconstrained icc analysis...

Second, exploratory testing of icc's...
icc; 

Third, confirmatory testing of icc's...
icc=0; 
icc>0; 

Call:
BF.lmerMod(x = icc_mod_infor, hypothesis = "icc = 0; icc > 0")

Bayesian hypothesis test
Type: confirmatory
Object: lmerMod
Parameter: intraclass correlations
Method: Bayes factors based on uniform priors

Posterior probabilities:
   Pr(hypothesis|data)
H1               0.012
H2               0.988
H3               0.000

Evidence matrix (Bayes factors):
       H1    H2  H3
H1  1.000 0.012 Inf
H2 85.355 1.000 Inf
H3  0.000 0.000   1

Hypotheses:
H1: icc=0
H2: icc>0
H3: complement

BF(icc_mod_value, hypothesis = "icc = 0; icc > 0")

First, unconstrained icc analysis...

Second, exploratory testing of icc's...
icc; 

Third, confirmatory testing of icc's...
icc=0; 
icc>0; 

Call:
BF.lmerMod(x = icc_mod_value, hypothesis = "icc = 0; icc > 0")

Bayesian hypothesis test
Type: confirmatory
Object: lmerMod
Parameter: intraclass correlations
Method: Bayes factors based on uniform priors

Posterior probabilities:
   Pr(hypothesis|data)
H1               0.006
H2               0.994
H3               0.000

Evidence matrix (Bayes factors):
        H1    H2  H3
H1   1.000 0.006 Inf
H2 174.388 1.000 Inf
H3   0.000 0.000   1

Hypotheses:
H1: icc=0
H2: icc>0
H3: complement

Perceived Informativity

Graphical Overview

study1_w %>% 
    ggplot(aes(type, infor)) +
    geom_jitter() +
    stat_summary(fun.data = mean_sdl,
                 fun.args = list(mult = 1),
                 color = "white") +
    theme_modern_rc() +
    labs(title = "Informativity",
         subtitle = "per Plot Type",
         caption = "Means ± 1*SD")

ICCs

Perceived Practical Relevance

Graphical Overview

study1_w %>% 
    ggplot(aes(type, value)) +
    geom_jitter() +
    stat_summary(fun.data = mean_sdl,
                 fun.args = list(mult = 1),
                 color = "white") +
    theme_modern_rc() +
    labs(title = "Informativity",
         subtitle = "per Plot Type",
         caption = "Means ± 1*SD")

ICCs

Sensitivity

ggplot(study1_w %>% 
           select(sensi_ordinal, type) %>% 
           na.omit(), aes(sensi_ordinal, 
                          color = sensi_ordinal,
                          fill = sensi_ordinal)) +
    facet_wrap(~ type) +
    geom_bar() +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white")) +
    ggtitle("Sensitivity", "all participants")

ggplot(study1_w %>% 
           select(sensi_ordinal, type, session) %>% 
           na.omit() %>% 
           filter(!session %in% ov_misconceptualizers & 
                      !session %in% u3_misconceptualizers), aes(sensi_ordinal, 
                          color = sensi_ordinal,
                          fill = sensi_ordinal)) +
    facet_wrap(~ type) +
    geom_bar() +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white")) +
    ggtitle("Sensitivity", "without missconceptualizers")

ggplot(study1_w %>% 
           select(sensi_binary_filtered, type, session),
       aes(sensi_binary_filtered, 
           color = sensi_binary_filtered,
           fill = sensi_binary_filtered)) +
    facet_wrap(~ type) +
    geom_bar() +
    theme_modern_rc() +
    theme(strip.text = element_text(color = "white")) +
    ggtitle("Binary Sensitivity", "without wrong decisions")

Warning: Removed 27 rows containing non-finite values (`stat_count()`).

Warning: The following aesthetics were dropped during statistical transformation:
colour, fill
ℹ This can happen when ggplot fails to infer the correct grouping structure in
  the data.
ℹ Did you forget to specify a `group` aesthetic or to convert a numerical
  variable into a factor?
The following aesthetics were dropped during statistical transformation:
colour, fill
ℹ This can happen when ggplot fails to infer the correct grouping structure in
  the data.
ℹ Did you forget to specify a `group` aesthetic or to convert a numerical
  variable into a factor?
The following aesthetics were dropped during statistical transformation:
colour, fill
ℹ This can happen when ggplot fails to infer the correct grouping structure in
  the data.
ℹ Did you forget to specify a `group` aesthetic or to convert a numerical
  variable into a factor?
The following aesthetics were dropped during statistical transformation:
colour, fill
ℹ This can happen when ggplot fails to infer the correct grouping structure in
  the data.
ℹ Did you forget to specify a `group` aesthetic or to convert a numerical
  variable into a factor?

Fitting a series of logistic regressions with non-informative priors

logreg_mod0 <- brm(sensi_binary_filtered ~ + (1|session), 
                    family = bernoulli(link = "logit"),
                    data = study1_w,
                    save_pars = save_pars(all = TRUE),
                   silent = 2,
                   refresh = 0)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

Running /Library/Frameworks/R.framework/Resources/bin/R CMD SHLIB foo.c
clang -arch arm64 -I"/Library/Frameworks/R.framework/Resources/include" -DNDEBUG   -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/Rcpp/include/"  -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/"  -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/unsupported"  -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/BH/include" -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/StanHeaders/include/src/"  -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/StanHeaders/include/"  -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppParallel/include/"  -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/rstan/include" -DEIGEN_NO_DEBUG  -DBOOST_DISABLE_ASSERTS  -DBOOST_PENDING_INTEGER_LOG2_HPP  -DSTAN_THREADS  -DUSE_STANC3 -DSTRICT_R_HEADERS  -DBOOST_PHOENIX_NO_VARIADIC_EXPRESSION  -DBOOST_NO_AUTO_PTR  -include '/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/StanHeaders/include/stan/math/prim/fun/Eigen.hpp'  -D_REENTRANT -DRCPP_PARALLEL_USE_TBB=1   -I/opt/R/arm64/include   -fPIC  -falign-functions=64 -Wall -g -O2  -c foo.c -o foo.o
In file included from <built-in>:1:
In file included from /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/StanHeaders/include/stan/math/prim/fun/Eigen.hpp:22:
In file included from /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/Eigen/Dense:1:
In file included from /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/Eigen/Core:88:
/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/Eigen/src/Core/util/Macros.h:628:1: error: unknown type name 'namespace'
namespace Eigen {
^
/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/Eigen/src/Core/util/Macros.h:628:16: error: expected ';' after top level declarator
namespace Eigen {
               ^
               ;
In file included from <built-in>:1:
In file included from /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/StanHeaders/include/stan/math/prim/fun/Eigen.hpp:22:
In file included from /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/Eigen/Dense:1:
/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/Eigen/Core:96:10: fatal error: 'complex' file not found
#include <complex>
         ^~~~~~~~~
3 errors generated.
make: *** [foo.o] Error 1

logreg_mod1 <- brm(sensi_binary_filtered ~ effsize_abs + (1|session), 
                   family = bernoulli(link = "logit"),
                   data = study1_w,
                   save_pars = save_pars(all = TRUE),
                   silent = 2,
                   refresh = 0)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

Running /Library/Frameworks/R.framework/Resources/bin/R CMD SHLIB foo.c
clang -arch arm64 -I"/Library/Frameworks/R.framework/Resources/include" -DNDEBUG   -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/Rcpp/include/"  -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/"  -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/unsupported"  -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/BH/include" -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/StanHeaders/include/src/"  -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/StanHeaders/include/"  -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppParallel/include/"  -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/rstan/include" -DEIGEN_NO_DEBUG  -DBOOST_DISABLE_ASSERTS  -DBOOST_PENDING_INTEGER_LOG2_HPP  -DSTAN_THREADS  -DUSE_STANC3 -DSTRICT_R_HEADERS  -DBOOST_PHOENIX_NO_VARIADIC_EXPRESSION  -DBOOST_NO_AUTO_PTR  -include '/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/StanHeaders/include/stan/math/prim/fun/Eigen.hpp'  -D_REENTRANT -DRCPP_PARALLEL_USE_TBB=1   -I/opt/R/arm64/include   -fPIC  -falign-functions=64 -Wall -g -O2  -c foo.c -o foo.o
In file included from <built-in>:1:
In file included from /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/StanHeaders/include/stan/math/prim/fun/Eigen.hpp:22:
In file included from /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/Eigen/Dense:1:
In file included from /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/Eigen/Core:88:
/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/Eigen/src/Core/util/Macros.h:628:1: error: unknown type name 'namespace'
namespace Eigen {
^
/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/Eigen/src/Core/util/Macros.h:628:16: error: expected ';' after top level declarator
namespace Eigen {
               ^
               ;
In file included from <built-in>:1:
In file included from /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/StanHeaders/include/stan/math/prim/fun/Eigen.hpp:22:
In file included from /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/Eigen/Dense:1:
/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/Eigen/Core:96:10: fatal error: 'complex' file not found
#include <complex>
         ^~~~~~~~~
3 errors generated.
make: *** [foo.o] Error 1

logreg_mod2 <- brm(sensi_binary_filtered ~ effsize_abs + type + (1|session), 
                   family = bernoulli(link = "logit"),
                   data = study1_w,
                   save_pars = save_pars(all = TRUE),
                   silent = 2,
                   refresh = 0)

Warning: Rows containing NAs were excluded from the model.

Trying to compile a simple C file

Running /Library/Frameworks/R.framework/Resources/bin/R CMD SHLIB foo.c
clang -arch arm64 -I"/Library/Frameworks/R.framework/Resources/include" -DNDEBUG   -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/Rcpp/include/"  -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/"  -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/unsupported"  -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/BH/include" -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/StanHeaders/include/src/"  -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/StanHeaders/include/"  -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppParallel/include/"  -I"/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/rstan/include" -DEIGEN_NO_DEBUG  -DBOOST_DISABLE_ASSERTS  -DBOOST_PENDING_INTEGER_LOG2_HPP  -DSTAN_THREADS  -DUSE_STANC3 -DSTRICT_R_HEADERS  -DBOOST_PHOENIX_NO_VARIADIC_EXPRESSION  -DBOOST_NO_AUTO_PTR  -include '/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/StanHeaders/include/stan/math/prim/fun/Eigen.hpp'  -D_REENTRANT -DRCPP_PARALLEL_USE_TBB=1   -I/opt/R/arm64/include   -fPIC  -falign-functions=64 -Wall -g -O2  -c foo.c -o foo.o
In file included from <built-in>:1:
In file included from /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/StanHeaders/include/stan/math/prim/fun/Eigen.hpp:22:
In file included from /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/Eigen/Dense:1:
In file included from /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/Eigen/Core:88:
/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/Eigen/src/Core/util/Macros.h:628:1: error: unknown type name 'namespace'
namespace Eigen {
^
/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/Eigen/src/Core/util/Macros.h:628:16: error: expected ';' after top level declarator
namespace Eigen {
               ^
               ;
In file included from <built-in>:1:
In file included from /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/StanHeaders/include/stan/math/prim/fun/Eigen.hpp:22:
In file included from /Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/Eigen/Dense:1:
/Library/Frameworks/R.framework/Versions/4.2-arm64/Resources/library/RcppEigen/include/Eigen/Core:96:10: fatal error: 'complex' file not found
#include <complex>
         ^~~~~~~~~
3 errors generated.
make: *** [foo.o] Error 1

bayes_factor(logreg_mod0, logreg_mod1)

Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 7

Estimated Bayes factor in favor of logreg_mod0 over logreg_mod1: 0.00000

bayes_factor(logreg_mod1, logreg_mod2)

Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Iteration: 7
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4

Estimated Bayes factor in favor of logreg_mod1 over logreg_mod2: 0.50420

sjPlot::tab_model(logreg_mod0, logreg_mod1, logreg_mod2, show.icc = TRUE)

	sensi_binary_filtered		sensi_binary_filtered		sensi_binary_filtered
Predictors	Odds Ratios	CI (95%)	Odds Ratios	CI (95%)	Odds Ratios	CI (95%)
Intercept	7.73	5.18 – 12.47	1.70	0.96 – 3.15	1.43	0.76 – 2.92
effsize_abs			37.00	14.81 – 91.89	38.12	14.82 – 100.96
typehalfeye_xaxis					1.55	0.87 – 2.73
typehalfeye_yaxis					1.35	0.77 – 2.39
typeraincloud_yaxis					0.95	0.55 – 1.65
Random Effects
σ2	3.29		3.29		3.29
τ00	1.32 session		1.64 session		1.70 session
ICC	0.29		0.33		0.34
N	40 session		40 session		40 session
Observations	933		933		933
Marginal R2 / Conditional R2	0.000 / 0.115		0.049 / 0.201		0.056 / 0.208

pp_check(logreg_mod2) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

Using 10 posterior draws for ppc type 'dens_overlay' by default.

Scale for colour is already present.
Adding another scale for colour, which will replace the existing scale.

Effectivity

Visualisation

Raw data

ggplot(study1_w_timestamp, aes(as.factor(plotNrWithin), effic)) +
    geom_boxplot(alpha = .2, color = "lightgrey") +
    geom_sina(alpha = .5) +
    coord_cartesian(ylim = c(0,100000)) + 
    facet_wrap(~type) +
    theme_modern_rc() +
    labs(title ="Dwell Times Until First Decision",
          subtitle = "Per Plot Type and Plot Repetition") +
    theme(strip.text = element_text(color = "white"))

5% Percent Truncated

ggplot(study1_w_timestamp, aes(as.factor(plotNrWithin), effic_05righttrunc)) +
    geom_boxplot(alpha = .2, color = "lightgrey") +
    geom_sina(alpha = .5) +
    coord_cartesian(ylim = c(0,85000)) + 
    facet_wrap(~type) +
    theme_modern_rc() +
    labs(title ="5% Truncated Dwell Times Until First Decision",
          subtitle = "Per Plot Type and Plot Repetition") +
    theme(strip.text = element_text(color = "white"))

Warning: Removed 48 rows containing non-finite values (`stat_boxplot()`).

Warning: Removed 48 rows containing non-finite values (`stat_sina()`).

10% Percent Truncated

ggplot(study1_w_timestamp, aes(as.factor(plotNrWithin), effic_10righttrunc)) +
    geom_boxplot(alpha = .2, color = "lightgrey") +
    geom_sina(alpha = .5) +
    coord_cartesian(ylim = c(0,85000)) + 
    facet_wrap(~type) +
    theme_modern_rc() +
    labs(title ="10% Truncated Dwell Times Until First Decision",
          subtitle = "Per Plot Type and Plot Repetition") +
    theme(strip.text = element_text(color = "white"))

Warning: Removed 96 rows containing non-finite values (`stat_boxplot()`).

Warning: Removed 96 rows containing non-finite values (`stat_sina()`).

log() Transformed

ggplot(study1_w_timestamp, 
       aes(as.factor(plotNrWithin), 
           log(effic_05righttrunc))) +
    geom_boxplot(alpha = .2, color = "lightgrey") +
    geom_sina(alpha = .5) +
    facet_wrap(~type) +
    theme_modern_rc() +
    labs(title ="log Transformed Dwell Times Until First Decision",
          subtitle = "Per Plot Type and Plot Repetition") +
    theme(strip.text = element_text(color = "white"))

Warning: Removed 48 rows containing non-finite values (`stat_boxplot()`).

Warning: Removed 48 rows containing non-finite values (`stat_sina()`).

Modeling with {brms}

Efficacy for First Plot

plot01_mod0 <- 
    brm(effic | trunc(lb = 0) ~ 1 + (1|session),
    data = study1_w_timestamp %>% 
        filter(plotNrWithin == 1),
    family = "student",
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    iter = 15000)

pp_check(plot01_mod0) +
    xlim(c(0,250000)) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

plot01_mod1 <- 
    brm(effic | trunc(lb = 0) ~ 1 + type + (1|session),
    data = study1_w_timestamp %>% 
        filter(plotNrWithin == 1),
    family = "student",
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    iter = 15000)

pp_check(plot01_mod1) +
    xlim(c(0,250000)) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

sjPlot::tab_model(plot01_mod0, plot01_mod1)

Warning: Following potential variables could not be found in the data: lb = 0

Warning: Following potential variables could not be found in the data: lb = 0

Warning: Following potential variables could not be found in the data: lb = 0

Warning: Following potential variables could not be found in the data: lb = 0

Warning: Following potential variables could not be found in the data: lb = 0

Warning: Following potential variables could not be found in the data: lb = 0

Warning: Following potential variables could not be found in the data: lb = 0

Warning: Following potential variables could not be found in the data: lb = 0

	Dependent variable		Dependent variable
Predictors	Estimates	CI (95%)	Estimates	CI (95%)
Intercept	24005.86	19032.63 – 28624.78	33257.96	27149.11 – 39476.42
typehalfeye_xaxis			-10495.52	-18175.36 – -3043.87
typehalfeye_yaxis			-12890.18	-19899.41 – -6140.58
typeraincloud_yaxis			-12604.36	-19859.71 – -5804.51
Random Effects
σ2	12080.20		10311.25
τ00	116188945.01 session		127794483.32 session
ICC	1.00		1.00
N	40 session		40 session
Observations	160		160
Marginal R2 / Conditional R2	0.000 / 0.016		0.006 / 0.027

Efficacy for Last Three Plots

plot0306_mod0 <- 
    brm(effic | trunc(lb = 0) ~ 1 + plotNrWithin_factor + (1|session),
    data = study1_w_timestamp %>% 
        filter(plotNrWithin >= 4),
    family = "student",
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    iter = 15000)

pp_check(plot0306_mod0) +
    xlim(c(0,250000)) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

plot0306_mod1 <- 
    brm(effic | trunc(lb = 0) ~ 1 + plotNrWithin_factor + type + (1|session),
    data = study1_w_timestamp %>% 
        filter(plotNrWithin >= 4),
    family = "student",
    save_pars = save_pars(all = TRUE),
    silent = 2,
    refresh = 0,
    iter = 35000,
    cores = 8)

pp_check(plot0306_mod1) +
    xlim(c(0,250000)) +
    theme_modern_rc() +
    scale_color_manual(values=viridis(2, begin = .3))

sjPlot::tab_model(plot0306_mod0, plot0306_mod1)

Warning: Following potential variables could not be found in the data: lb = 0

Warning: Following potential variables could not be found in the data: lb = 0

Warning: Following potential variables could not be found in the data: lb = 0

Warning: Following potential variables could not be found in the data: lb = 0

Warning: Following potential variables could not be found in the data: lb = 0

Warning: Following potential variables could not be found in the data: lb = 0

Warning: Following potential variables could not be found in the data: lb = 0

Warning: Following potential variables could not be found in the data: lb = 0

	Dependent variable		Dependent variable
Predictors	Estimates	CI (95%)	Estimates	CI (95%)
Intercept	11760.77	9729.31 – 13833.14	11318.57	-1136.85 – 14399.29
plotNrWithin_factor: plotNrWithin_factor5	-334.89	-1673.94 – 988.06	-148.07	-1494.53 – 1223.86
plotNrWithin_factor: plotNrWithin_factor6	-1525.21	-2829.93 – -288.15	-1359.00	-2610.05 – -127.29
typehalfeye_xaxis			-529.35	-2156.93 – 1274.83
typehalfeye_yaxis			-1536.46	-3082.73 – 6.55
typeraincloud_yaxis			-278.14	-1738.57 – 1201.78
Random Effects
σ2	4066.88		3786.55
τ00	30721907.53 session		73643292.46 session
ICC	1.00		1.00
N	40 session		40 session
Observations	480		480
Marginal R2 / Conditional R2	0.000 / 0.178		0.000 / 0.382

References

Arslan, Ruben C., Matthias P. Walther, and Cyril S. Tata. 2020. “Formr: A Study Framework Allowing for Automated Feedback Generation and Complex Longitudinal Experience-Sampling Studies Using R.” Behavior Research Methods 52 (1): 376–87. https://doi.org/10.3758/s13428-019-01236-y.